# -*- coding:'utf-8' -*-
'''Export LineDatas(json style) to Excel'''
'''Create by Sidchen in 2022'''

import xlsxwriter
import json
import os
from datetime import datetime
import math
import shutil
import re


# 容错判断
def havefile(file):
    if os.path.exists(file):
        print('###########已找到' + file + '，解析中…………')
        return process(file)
    else:
        input('未找到' + file + '，请确认，任意键退出')
        exit()


# json文件的处理
def process(file):
    # 去掉文件的BOM(ByteOrderMark)，否则loads会报错
    fo = open(file, 'r', encoding='utf-8')
    data = fo.read()
    fo.close()
    if data.startswith(u'\ufeff'):
        data = data.encode('utf8')[3:].decode('utf8')
    # 将多个对象拆解成数组,并去除原始json中的线路信息重复
    k = data.strip('[]').split('},{')
    if len(k) != 1:
        h = set()
        h.add(k[0] + '}')
        for i in range(len(k) - 2):
            h.add('{' + k[i + 1] + '}')
        h.add('{' + k[-1])
        # 避免无法识别包含{}的set
        js = []
        for g in h:
            js.append(g)
        return js
    else:
        #only one line info
        return k


# 解析json并写入表格
def save(head, data, name):
    # 获取当前日期
    today = datetime.today()
    # 将获取到的datetime对象仅取日期
    today_date = datetime.date(today)
    # 创建表格
    f = name + str(today_date) + '.xlsx'
    table = xlsxwriter.Workbook(f)
    sheet = table.add_worksheet('sheet1')
    # 写入表头
    for i in range(len(head)):
        sheet.write(0, i, str(head[i]))
    # 写入数据
    for i in range(len(data)):
        try:
            for j in range(len(head)):
                # k = json.loads(data[i])
                k = data[i]
                if str(k[head[j]]) == '[]':
                    sheet.write(i + 1, j, '')
                else:
                    if is_number(k[head[j]]):
                        sheet.write(i + 1, j, float(k[head[j]]))
                    else:
                        sheet.write(i + 1, j, str(k[head[j]]))
        except:
            wrong.append(('目标字段不完整', name, data[i]))
    table.close()
    print('已生成：' + f)


# 简单判断是否是数字
def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        pass
    return False


# 去掉保存文件的非法字符
def valid_name(title):
    # '/ \ : * ? " < > |'
    invalid = r"[\/\\\:\*\?\"\<\>\|]"
    # 替换为下划线
    new_title = re.sub(invalid, "-", title)
    return new_title


# 提取每条线路的站点、线路点位信息
def getline(name, data):
    line = []
    for i in data:
        d = json.loads(i)
        if d['linename'] == name:
            # 转换火星坐标为WGS84坐标
            d['lng'], d['lat'] = gcj02towgs84(d['lng'], d['lat'])
            line.append(d)
    return line


# 提取每条线路的站点、线路点位信息
def getline2(name, data):
    line = []
    for d in data:
        if d['linename'] == name:
            # 转换火星坐标为WGS84坐标
            d['lng'], d['lat'] = gcj02towgs84(d['lng'], d['lat'])
            line.append(d)
    return line


# 经纬度转换
x_pi = 3.14159265358979324 * 3000.0 / 180.0
pi = 3.1415926535897932384626  # π
a = 6378245.0  # 长半轴
ee = 0.00669342162296594323  # 扁率


def wgs84togcj02(lng, lat):
    """
    WGS84转GCJ02(火星坐标系)
    :param lng:WGS84坐标系的经度
    :param lat:WGS84坐标系的纬度
    :return:
    """
    if out_of_china(lng, lat):  # 判断是否在国内
        return lng, lat
    dlat = transformlat(lng - 105.0, lat - 35.0)
    dlng = transformlng(lng - 105.0, lat - 35.0)
    radlat = lat / 180.0 * pi
    magic = math.sin(radlat)
    magic = 1 - ee * magic * magic
    sqrtmagic = math.sqrt(magic)
    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
    mglat = lat + dlat
    mglng = lng + dlng
    return [mglng, mglat]


def gcj02towgs84(lng, lat):
    """
    GCJ02(火星坐标系)转GPS84
    :param lng:火星坐标系的经度
    :param lat:火星坐标系纬度
    :return:
    """
    if out_of_china(lng, lat):
        return lng, lat
    dlat = transformlat(lng - 105.0, lat - 35.0)
    dlng = transformlng(lng - 105.0, lat - 35.0)
    radlat = lat / 180.0 * pi
    magic = math.sin(radlat)
    magic = 1 - ee * magic * magic
    sqrtmagic = math.sqrt(magic)
    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
    mglat = lat + dlat
    mglng = lng + dlng
    return [lng * 2 - mglng, lat * 2 - mglat]


def transformlat(lng, lat):
    ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
          0.1 * lng * lat + 0.2 * math.sqrt(math.fabs(lng))
    ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
            math.sin(2.0 * lng * pi)) * 2.0 / 3.0
    ret += (20.0 * math.sin(lat * pi) + 40.0 *
            math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
    ret += (160.0 * math.sin(lat / 12.0 * pi) + 320 *
            math.sin(lat * pi / 30.0)) * 2.0 / 3.0
    return ret


def transformlng(lng, lat):
    ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
          0.1 * lng * lat + 0.1 * math.sqrt(math.fabs(lng))
    ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
            math.sin(2.0 * lng * pi)) * 2.0 / 3.0
    ret += (20.0 * math.sin(lng * pi) + 40.0 *
            math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
    ret += (150.0 * math.sin(lng / 12.0 * pi) + 300.0 *
            math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
    return ret


def out_of_china(lng, lat):
    """
    判断是否在国内，不在国内不做偏移
    :param lng:
    :param lat:
    :return:
    """
    if lng < 72.004 or lng > 137.8347:
        return True
    if lat < 0.8293 or lat > 55.8271:
        return True
    return False


# write data into txt
def txt_write(list_text, path, txt):
    ftxt = open(path, mode='w', encoding='utf8')
    if list_text == 'list':
        for i in txt:
            ftxt.write(str(i)+'\n')
    else:
        ftxt.write(txt)
    ftxt.close()


# end remark
def end():
    check = input_wait('\n* 检查脚本更新情况，稍候片刻…………\n（无需更新，可按Enter键退出）\n', 'checking......')
    if len(check) > 0:
        update_check(url_update, progress)
        # check and download the updated progress
        progress2 = {'RoutesDraw.tbx': ['20210320',
                                        'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=a3253036%2D41dd%2D4953%2Dbe15%2Db48a7eb1d176'
                                        '']
                     }
        # url must be the direct sharing link if it's from OneDrive where you can check latest update date
        url_update2 = 'https://chdeducn-my.sharepoint.com/:f:/g/personal/201541020106_chd_edu_cn/Eopn9g6GHeNDkkyd09Jp3RMBKcrwSKaj5eEWdMyXlGxXEw?e=KMcgKX' \
                      ''
        update_check(url_update2, progress2)
    print('\n'
          '\n****************************'
          '\n*数据已全部获取，Enter键退出*'
          '\n****************************\n\n')
    input()
    exit()


def DownloadHtml(url):
    import string
    from urllib.parse import quote
    import requests
    import time
    from datetime import datetime
    # 伪装user-agent，因为联网不稳定所以避免使用fake_useragent模块
    # 参考：https://blog.csdn.net/weixin_44818729/article/details/107060218
    browsers = {
        "chrome": [
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.222"
            "7.1 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41."
                                 "0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
                                                           "(KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/53"
            "7.36", "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 S"
                    "afari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chro"
                                    "me/41.0.2225.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
            "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36",
            "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36",
            "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
            "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14"],
        "opera": ["Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16",
                  "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
                  "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
                  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
                  "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
                  "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
                  "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00",
                  "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
                  "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
                  "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0",
                  "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
                  "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
                  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
                  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52",
                  "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51",
                  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51",
                  "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50",
                  "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50",
                  "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11",
                  "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11",
                  "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11",
                  "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10",
                  "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10",
                  "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10",
                  "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1",
                  "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01",
                  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
                  "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
                  "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
                  "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01",
                  "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00"],
        "firefox": ["Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
                    "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0",
                    "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0",
                    "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0",
                    "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101  Firefox/28.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0",
                    "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0",
                    "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0",
                    "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0",
                    "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0",
                    "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0)  Gecko/20100101 Firefox/18.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6"],
        "internetexplorer": ["Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
                             "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0;  rv:11.0) like Gecko",
                             "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)",
                             "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)",
                             "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
                             "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)",
                             "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
                             "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)",
                             "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)"],
        "safari": [
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
            "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
            "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5"]}
    from random import choice
    ag = choice(browsers[choice(list(browsers.keys()))])
    headers = {
        # 伪装
        'User-Agent': ag
    }
    # 请求
    # 解决请求路径中含义中文或特殊字符
    url = quote(url, safe=string.printable)
    try:
        request = requests.get(url=url, headers=headers, timeout=(5, 27))
        html = request.text
        request.close()
        return html
    except:
        print('{}网页响应错误，30s后自动无限重试……\n（如果疯狂重试不成功，请手动关闭脚本，确认账号后再运行）'.format(datetime.now()))
        time.sleep(30)
        DownloadHtml(url)


def update_check(url, progress):
    import re
    from datetime import datetime
    import webbrowser
    from time import sleep

    # check update
    progress_ud = {}
    # progress_ud = {'name':[version_new, version_old, interval]}
    html = DownloadHtml(url)
    for i in progress:
        print(f'正在检查更新：{i}')
        # progress = {'name': ['version_old', 'download_link']}
        try:
            date = re.search(f'{i}.*"Modified.": "', html, re.S).span()[1]
            date_update = html[date:date+10].replace('-', '')
            version_new = datetime.strptime(date_update, '%Y%m%d')
            version_old = datetime.strptime(progress[i][0], '%Y%m%d')
            interval = version_new-version_old
            if interval.days > 0:
                progress_ud[f'{i}'] = (date_update, progress[i][0], interval)
                print(f'# 有更新:{date_update} ')
        except:
            print('* 未知错误')
    if len(progress_ud) > 0:
        load = input_wait('需要调起默认浏览器下载最新脚本文件吗？\n(默认需要，Enter退出)\n', 'Yes')
        if len(load) > 0:
            for file in progress_ud:
                # start the default browser to download the new version
                webbrowser.open(progress[file][1], new=2)
    else:
        print('* 无更新内容')


def input_wait(remind, defaut):
    """input or leap over"""
    # CODE from: https://blog.csdn.net/weixin_39858881/article/details/107152961
    import func_timeout

    @func_timeout.func_set_timeout(5)
    def askChoice():
        return input(remind)

    try:
        s = askChoice()
    except func_timeout.exceptions.FunctionTimedOut as e:
        s = defaut
    print('\n', s)
    return s


# 主程序
if __name__ == '__main__':
    print('* 本脚本用于析出目标公交线路数据')
    # check and download the updated progress
    progress = {'1-ExtracBusLines.exe': ['20210419',
                                         'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=e11e5abc%2D713f%2D499c%2Db0a8%2D77900610974d'
                                         ]
        , '2-GetBusRoutes.html': ['20220906',
                                  'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=b1307354%2Dc513%2D4f4a%2Db5f6%2Da62033dfd587'
                                  ]
        , '3-LineDataToGIS.exe': ['20220906',
                                  'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=3e788dbb%2D596f%2D4507%2Db0dc%2D9ed0eb664f09'
                                  ]
        , 'UpdateLogs.txt': ['20220906',
                             'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=5409cb08%2De55f%2D478c%2Daa57%2Df7b1f1484e32'
                             ]
                }
    # url must be the direct sharing link if it's from OneDrive where you can check latest update date
    url_update = 'https://chdeducn-my.sharepoint.com/:f:/g/personal/201541020106_chd_edu_cn/EsDls6kHw_dEjvkurmGVbEYBQWCkemLsnMYdltXlhnV24w?e=Eel0Q0' \
                 ''
    for i in progress:
        print(f'* {i}，版本号{progress[i][0]}，')
    print('*** 有限技术支持：sidchen0 @ qq.com ***\n\n')

    print('为避免重复写入RoutesGet文件夹，将自动删除')
    try:
        if os.path.exists('RoutesGet/'):
            shutil.rmtree('RoutesGet/')
    except:
        input('RoutesGet文件夹下可能有文件打开，请关闭后再运行脚本。Endter键推出')
        exit()
    if not os.path.exists('RoutesGet/LineDraw'):
        os.makedirs('RoutesGet/LineDraw')

    # 储存未知错误的线路数据
    wrong = [('错误原因', '文件名称', '原数据')]

    # 读取储存线路信息总表t1.sheet1
    bl = havefile('Linedata/BusLines.json')
    # 为了合并储存
    bl2 = []
    # 将name,stops.name,stops.sequence,stops.location.lng/lat单独储存为每条线路的沿途站点表t[x].sheet1
    # 将path储存为线路定位表t[x].sheet2
    bs = havefile('Linedata/BusStops.json')
    dl = havefile('Linedata/DrawLines.json')
    headbs = ['linename', 'stopname', 'sequence', 'lng', 'lat']
    headdl = ['linename', 'order', 'lng', 'lat']
    all_lines = []
    all_stops = []
    # 储存环路的情况，即不在往返对比数据组里
    circular = []

    # 完整线路总表，通过公交线路查询获得
    lines = []
    for i in range(len(bl)):
        ln = json.loads(bl[i])
        lines.append(ln['linename'])

    # 先检测是否有进行往返对比
    file_compare = 'Linedata/LinesDataCompare.json'
    if os.path.exists(file_compare):
        print('*#* 识别到公交线路往返对比文件')
        # 去掉文件的BOM(ByteOrderMark)，否则loads会报错
        fo = open(file_compare, 'r', encoding='utf-8')
        data = fo.read()
        fo.close()
        if data.startswith(u'\ufeff'):
            data = data.encode('utf8')[3:].decode('utf8')
        data_compare = json.loads(data)
        # 储存往返比对结果
        LineNames = {}
        for i in range(0, len(data_compare)-1, 2):
            # 进行往返比对：所有途径站点应该一致
            name0 = data_compare[i]['linename'].split('(')[0]
            on1 = data_compare[i]['start']
            off1 = data_compare[i]['end']
            on2 = data_compare[i+1]['start']
            off2 = data_compare[i+1]['end']
            print(on1, off1)
            print(on2, off2)
            tick = '往返一致'
            # 起终点检验
            if (on1 not in [on2, off2]) or (off1 not in [on2, off2]):
                tick = '起终点不同'
            # 途径站点检验
            stops1 = []
            for j in data_compare[i]['stops']:
                stops1.append(j['stopname'])
            tick2 = 0
            for j in data_compare[i+1]['stops']:
                if j['stopname'] not in stops1:
                    tick2 += 1
            if tick2 != 0:
                tick = tick+' '+str(tick2)+'个途径站点不同'
            print('*{}公交往返线路不同：{}'.format(name0, tick))
            # i用来定位不一致的线路
            LineNames[name0] = [tick, i]
        print('(如果往返线路检测有误，相信我是高德数据源的问题…………)')
        for j in bl:
            # 总表
            j = json.loads(j)
            name2 = j['linename'].split('(')[0]
            # 合并总表、往返检验表
            # 环路，起终点一致，未被往返对比收录
            if j['start'] == j['end']:
                j['DirectionCheck'] = '环路'
                bl2.append(j)
                circular.append(j['linename'])
            # 存在往返线路的数据：包括往返一致和不一致
            else:
                k = LineNames[name2]
                if k[0] == '往返一致':
                    # 往返一致：储存总表中的线路
                    j['DirectionCheck'] = '往返一致'
                    bl2.append(j)
                else:
                    # 往返不一致，储存总表数据一条+往返表中另一条（无其他详细信息）
                    j['DirectionCheck'] = k[0]
                    bl2.append(j)
                    # 另一条
                    h = data_compare[k[1]]
                    if j['linename'] == h['linename']:
                        h = data_compare[k[1]+1]
                    line2 = {'linename': h['linename'], 'start': h['start'], 'end': h['end'],'stime': j['stime'],
                             'etime': j['etime'], 'tprice': j['tprice'],'bprice': j['bprice'], 'dis': '',
                             'DirectionCheck': k[0]}
                    bl2.append(line2)
        # 储存线路总表：字段名称是json文件中的对象key名称
        head = ['linename', 'start', 'end', 'stime', 'etime', 'tprice', 'bprice', 'dis', 'DirectionCheck']
        save(head, bl2, 'RoutesGet/BusLinesInfo')

        # 储存线路站点总表
        for i in circular:
            # 转换站点点位，路径点位坐标系
            e = getline(i, bs)
            f = getline(i, dl)
            all_stops.append(e)
            all_lines.append(f)
            # 去掉线路中的非法字符
            LineName = valid_name(i)
            save(headbs, e, 'RoutesGet/LineDraw/' + i + 'Stops')
            save(headdl, f, 'RoutesGet/LineDraw/' + i + 'PathDraw')
            print('最大共 {} 条线路，已生成 {} 条环路，剩余 {} 条'.format(len(lines), len(circular), len(lines)-len(circular)))
        tick = len(circular)
        for i in data_compare:
            name0 = i['linename']
            for j in i['stops']:
                j['linename'] = name0
            for k in i['pathdraw']:
                k['linename'] = name0
            e2 = getline2(name0, i['stops'])
            f2 = getline2(name0, i['pathdraw'])
            all_stops.append(e2)
            all_lines.append(f2)
            save(headbs, e2, 'RoutesGet/LineDraw/' + name0 + 'Stops')
            save(headdl, f2, 'RoutesGet/LineDraw/' + name0 + 'PathDraw')
            tick += 1
            print('最大共 {} 条线路，已生成 {} 条环路，剩余 {} 条'.format(len(lines), tick, len(lines)-tick))
    else:
        # 总表
        for i in bl:
            bl2.append(json.loads(i))
        # 字段名称是json文件中的对象key名称
        head = ['linename', 'start', 'end', 'stime', 'etime', 'tprice', 'bprice', 'dis']
        save(head, bl2, 'RoutesGet/BusLinesInfo')
        # 站点表、路径表
        for i in range(len(lines)):
            e = getline(lines[i], bs)
            f = getline(lines[i], dl)
            all_stops.append(e)
            all_lines.append(f)
            # 去掉线路中的非法字符
            LineName = valid_name(lines[i])
            save(headbs, e, 'RoutesGet/LineDraw/' + LineName + 'Stops')
            save(headdl, f, 'RoutesGet/LineDraw/' + LineName + 'PathDraw')
            print('最大共 {} 条线路，已生成 {} 条线路，剩余 {} 条'.format(len(lines), i+1, len(lines)-i-1))
    print('\n\n将所有线路转存到同一张表中，便于ArcGIS中的后续操作')
    all_lines2 = []
    all_stops2 = []
    for i in all_lines:
        for j in i:
            all_lines2.append(j)
    for i in all_stops:
        for j in i:
            all_stops2.append(j)
    save(headdl, all_lines2, 'RoutesGet/AllBusLines')
    save(headbs, all_stops2, 'RoutesGet/AllBusStops')

    if len(wrong) > 1:
        print('\n发现{}处数据不完整'.format(len(wrong)-1))
        txt_write('list', 'RoutesGet/wrong.txt', wrong)
    end()
